package com.gxljc.bear.crawler.itaogao.yicai;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.Map;

import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.log4j.Logger;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.gxljc.commons.util.Pair;
import com.gxljc.commons.util.StringUtil;
import com.gxljc.bear.crawler.base.DolphinCrawlerConsts;
import com.gxljc.bear.crawler.itaogao.NewsBaseCrawler;
import com.gxljc.bear.crawler.itaogao.NewsBaseTable;

public class YiCaiDetailCrawler extends NewsBaseCrawler {
	/**
	 *
	 */
	private static final long serialVersionUID = 2193221959567325686L;
	private final static Logger LOG = Logger.getLogger(YiCaiDetailCrawler.class);
	private final static String RESTFUL_URL_FORMAT = "http://m.yicai.com/ajax/oneList/77/200/%s"; // '77':新闻频道;'200':pageSize
	private static final int MAX_PAGE = 90; // total = 200 * 90
	private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36";
	private static final String COOKIE = "yu_id=4f54e2cb7881451982df293b9fa01bd4; _gat=1; cn_ff152588baaf1u0970f0_dplus=%7B%22distinct_id%22%3A%20%22156fe210e478b-0c3729a0b-4349052c-1fa400-156fe210e485b8%22%2C%22%24_sessionid%22%3A%200%2C%22%24_sessionTime%22%3A%201473153497%2C%22%24dp%22%3A%200%2C%22%24_sessionPVTime%22%3A%201473153497%2C%22initial_view_time%22%3A%20%221473139191%22%2C%22initial_referrer%22%3A%20%22%24direct%22%2C%22initial_referrer_domain%22%3A%20%22%24direct%22%2C%22%24recent_outside_referrer%22%3A%20%22%24direct%22%7D; Hm_lvt_80b762a374ca9a39e4434713ecc02488=1473142395; Hm_lpvt_80b762a374ca9a39e4434713ecc02488=1473153568; _ga=GA1.2.2023853117.1473143840";
	private static final String REFERER = "http://m.yicai.com/news/finance/";

	public YiCaiDetailCrawler() {
		super("t_site_yicai", DolphinCrawlerConsts.MongoDBName.ITAOGAO.getName());
	}

	@SuppressWarnings("unchecked")
	@Override
	protected List<String> genSeeds() {
		LOG.info("-- genSeeds of YiCai--");

		List<String> seeds = new ArrayList<>();
		for (int i = 1; i <= MAX_PAGE; i++) {
			// parse seeds from restful
			String request = String.format(RESTFUL_URL_FORMAT, i + "");
			String result = resultByHttpClient(request);
			if (!StringUtil.isNullOrEmpty(result)) {
				// System.out.println("result="+result);
				try {

					JSONObject jsonResult = (JSONObject) JSONObject.parse(result);
					if (jsonResult == null) {
						continue;
					}
					JSONArray jsonArray = (JSONArray) jsonResult.get("list");
					if (jsonArray != null) {
						for (Object object : jsonArray) {
							Map<String, String> m = (Map<String, String>) object;
							String url = m.get("url");
							if (!StringUtil.isNullOrEmpty(url)) {
								seeds.add(switchURL(url));
							}
						}
					}
				} catch (Exception e) {
					LOG.error("parse error for request=" + request);
				}
			}
		}
		LOG.info("seeds size=" + seeds.size());
		return seeds;
	}

	private String switchURL(String url) {
		return url.replaceAll("m.yicai.com", "www.yicai.com");
	}

	// public static void main(String[] args) throws Exception {
	// String request = "";
	// String result = "";
	// System.out.println(new YiCaiDetailCrawler().genSeeds());
	// }

	@SuppressWarnings("unchecked")
	@Override
	public Pair<NewsBaseTable, byte[]> genTable(String url) {
		NewsBaseTable table = new NewsBaseTable();
		String request = url;
		String result = "";
		Pair<String, byte[]> pair = fetResult(request, proxy);
		if (pair == null) {
			return new Pair<NewsBaseTable, byte[]>(table, "".getBytes());
		}
		result = pair.first;
		byte[] html = pair.second;

		if (result != null) {
			JSONObject jsonResult = (JSONObject) JSONObject.parse(result);
			if (jsonResult != null) {

				List<String> paraList = (List<String>) jsonResult.get("para_list");
				List<String> images = (List<String>) jsonResult.get("pic_list");
				String title = (String) jsonResult.get("title");
				String date = (String) jsonResult.get("date");
				String tag = (String) jsonResult.get("tag");
				String source = (String) jsonResult.get("source");
				String author = (String) jsonResult.get("author");
				if (!StringUtil.isNullOrEmpty(title)) {
					table.setTitle(title);
				}
				if (!StringUtil.isNullOrEmpty(date)) {
					table.setPublishDate(genDate(date, "yyyy-MM-dd HH:mm"));
				}
				if (!StringUtil.isNullOrEmpty(tag)) {
					table.setTagName(Arrays.asList(tag));
				}
				
				if (!StringUtil.isNullOrEmpty(author)) {
					table.setAuthor(author);
				}
				if(validateObject(source)){
					table.setSource(source);
				}
				if (validateObject(images)) {
					table = processImages(url, table, images);
				}
				if (paraList != null) {
					table.setContent(genContent(paraList, table.getImages()));
				}
				// set default value
				table.setId(StringUtil.genMD5Val(url));
				table.setCrawlUrl(url);
				table.setCrawlDate(new Date());
				table.setCrawlFlag(1);
			}
		}
		return new Pair<NewsBaseTable, byte[]>(table, html);
	}

	public String resultByHttpClient(String url) {
		try {
			return genGetMethod(url).getResponseBodyAsString();
		} catch (Exception e) {
			e.printStackTrace();
		}
		return null;
	}

	private GetMethod genGetMethod(String url) throws Exception {
		HttpClient httpClient = new HttpClient();
		GetMethod getMethod = new GetMethod(url);
		getMethod.setRequestHeader("User-Agent", USER_AGENT);
		getMethod.setRequestHeader("Cookie", COOKIE);
		getMethod.setRequestHeader("Referer", REFERER);
		getMethod.getParams().setContentCharset("UTF8");
		httpClient.executeMethod(getMethod);
		httpClient.getHttpConnectionManager().getParams().setConnectionTimeout(5000);
		httpClient.getHttpConnectionManager().getParams().setSoTimeout(20000);
		return getMethod;
	}

	@SuppressWarnings("unchecked")
	@Override
	protected List<String> genNewSeeds() {
		LOG.info("-- genNewSeeds of YiCai--");

		List<String> seeds = new ArrayList<>();
		// parse seeds from restful
		String request = String.format(RESTFUL_URL_FORMAT, 1 + "");
		String result = resultByHttpClient(request);
		if (!StringUtil.isNullOrEmpty(result)) {
			// System.out.println("result="+result);
			try {

				JSONObject jsonResult = (JSONObject) JSONObject.parse(result);
				if (jsonResult == null) {
					return seeds;
				}
				JSONArray jsonArray = (JSONArray) jsonResult.get("list");
				if (jsonArray != null) {
					for (Object object : jsonArray) {
						Map<String, String> m = (Map<String, String>) object;
						String url = m.get("url");
						if (!StringUtil.isNullOrEmpty(url)) {
							seeds.add(switchURL(url));
						}
					}
				}
			} catch (Exception e) {
				LOG.error("parse error for request=" + request);
			}
		}
		LOG.info("seeds size=" + seeds.size());
		return seeds;
	}

}
